﻿using System;
using System.Collections;
using System.Linq;
using System.Text;
using WebLibrary;
using System.IO;
using HtmlAgilityPack;

namespace PhoneBook
{
    class Program
    {
        static void Main(string[] args)
        {
            String keywords = String.Empty;
            String location = String.Empty;
            int pagesLimit = 10;

            bool ignorePTCL = Boolean.Parse(System.Configuration.ConfigurationSettings.AppSettings["IgnorePTCL"].ToString());

            WebSession session = new WebSession();
            String url = "http://phonebook.com.pk/Dynamic/Search.aspx?k=KEYWORD&l=LOCATION&SearchType=kl&page=PAGENO";

            String output = String.Empty;
            String responce = String.Empty;

            int total_number_found = 0;
            int ptcl_numbers = 0;
            int legal_numbers = 0;

            try
            {
                Console.Write("Enter Keyword(s) or Business Name: ");
                keywords = Console.ReadLine().ToString();

                System.Console.Write("Enter Location: ");
                location = Console.ReadLine().ToString();

                System.Console.Write("Enter Page Search Limit: ");
                pagesLimit = int.Parse(Console.ReadLine().ToString());

                String[] proxyList = GetWebProxies();
                int index = 1;
                int ProxyRetry = 3;

                for (int i = 1; i <= pagesLimit; i++)
                {
                    Console.WriteLine("");
                    Console.WriteLine("===> Page No " + i.ToString());

                    String postURL = url.Replace("KEYWORD", keywords).Replace("LOCATION", location).Replace("PAGENO", i.ToString());

                    if (ProxyRetry > 2)
                    {
                        session.Proxy = GetWorkingProxy(proxyList, ref index);
                        ProxyRetry = 0;
                    }
                    
                    Console.WriteLine("Post Request.");
                    session.Get(postURL);
                    Console.WriteLine("Get Response.");
                        
                    
                    HtmlAgilityPack.HtmlDocument hDoc = new HtmlAgilityPack.HtmlDocument();
                    HtmlAgilityPack.HtmlNodeCollection nodes = null;

                    hDoc.LoadHtml(session.Document.Html);
                    nodes = hDoc.DocumentNode.SelectNodes("//div[@class='rescont']//div[@class='resrow']//div[@class='resdet']//h2//span");

                    if (session.Document.Html.ToString().Contains("your query looks similar to automated requests"))
                    {
                        i = i - 1;
                        ProxyRetry = 3;
                        Console.WriteLine("Blocked by the site. Trying again for blocked page.");
                        Console.WriteLine("");
                        continue;
                    }

                    if (nodes == null)
                        break;

                    Console.WriteLine("");
                    Console.WriteLine("Contacts");
                    foreach (HtmlAgilityPack.HtmlNode node in nodes)
                    {
                        String number = node.InnerText;
                        number = number.Replace("-", "");
                        total_number_found++;

                        if (number.Length == 13)
                        {
                            if (ignorePTCL && number.StartsWith("+924"))
                            {
                                ptcl_numbers++;
                                continue;
                            }

                            Console.Write(number + "\n");
                            responce += number + "\r\n";
                            legal_numbers++;
                        }
                    }


                    if (i != pagesLimit)
                    {
                        Console.WriteLine("Sleep for 10 Seconds");
                        System.Threading.Thread.Sleep(10 * 1000);
                    }

                    ProxyRetry++;
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine("Exception: " + "\n" + ex.Message + "\n" + ex.StackTrace);
            }

            try
            {
                if (responce.Length > 0)
                    responce = responce.Remove(responce.Length - 1);
                else
                    throw new Exception("Failed to get any Contacts.");

                output = keywords + "_" + location + ".txt";


                if (!File.Exists(output))
                    File.Create(output).Close();

                File.WriteAllText(output, String.Empty);

                StreamWriter writer = new StreamWriter(output, true);
                writer.WriteLine(responce);

                writer.Flush();
                writer.Close();
            }
            catch (Exception ex)
            {
                Console.WriteLine("Exception: " + "\n" + ex.Message + "\n" + ex.StackTrace);
            }

            Console.WriteLine("");
            Console.WriteLine("Statistics");
            Console.WriteLine("Total Numbers on pages = " + total_number_found.ToString());
            Console.WriteLine("Legal numbers = " + legal_numbers.ToString());
            Console.WriteLine("PTCL numbers = " + ptcl_numbers.ToString());
            Console.WriteLine("");
            Console.Write("Press Enter To Exit.");
            Console.ReadLine();
        }

        private static System.Net.WebProxy GetWorkingProxy(String[] proxyList, ref int index)
        {
            WebSession session = new WebSession();

            Console.WriteLine("Checking Proxies...");

            while (true)
            {
                try
                {
                    session.Proxy = new System.Net.WebProxy(proxyList[index]);
                    session.Timeout = 15 * 1000;
                    Console.WriteLine ("Ping " + proxyList[index]);
                    session.Get("http://dreamincode.net");

                    break;
                }
                catch (Exception ex) 
                { index++; }
            }

            Console.WriteLine("Found Proxy: " + proxyList[index]);
            return new System.Net.WebProxy(proxyList[index]);
        }


        private static String[] GetWebProxies()
        {
            Console.WriteLine("Fetching Proxy List");

            String proxyURL = "http://premium.freeproxy.ru/en/proxies/txt/?type[]=HTTPS&type[]=SOCKS4&type[]=SOCKS5&anon[]=HIA&anon[]=ANM&anon[]=NOA&ports=&sort_by=trust&sort_order=desc&per_page=50&code=b8fe020acd3c7b295ad8560fa8db5f16";
            
            WebSession session = new WebSession();
            session.Get(proxyURL);

            String hDOC = session.Document.Html.ToString();
            String[] proxies = hDOC.Replace("\r", "").Split('\n');

            Console.WriteLine("Got Proxies.");
            Array.Reverse(proxies);
            return proxies;
        }
    }
}
